import time
import requests
import logging
from selenium import webdriver
from selenium.webdriver import Keys
from selenium.webdriver.common.by import By

option=webdriver.EdgeOptions()
option.add_experimental_option("detach",False)
driver=webdriver.Edge(options=option)
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s : %(message)s')

BASE_URL = 'https://www.chenxuhou.com'
TOTAL_PAGE = 1
driver.get(BASE_URL)
searchInput=driver.find_element(By.XPATH,'//*[@id="search"]/input')
searchInput.send_keys('redis')
searchInput.send_keys(Keys.ENTER)
time.sleep(3)

lists=driver.find_elements(By.CLASS_NAME,'content-list')
acticles=[]
for item in lists:
    title=item.find_element(By.TAG_NAME,'h3').text
    link=item.find_element(By.TAG_NAME,'a').get_attribute('href')
    print(title ,link)
    acticles.append({'title':title,'link':link,'content':''})

print(acticles)

for key,item in enumerate(acticles):
    driver.get(item['link'])
    content=driver.find_element(By.CLASS_NAME,'article-content').get_attribute('innerHTML')
    acticles[key]['content']=content
    print(content)
    break

print(acticles)
exit()
def scrape_page(url):
    logging.info('scraping %s...', url)
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
        logging.error('get url invalid code $s while scraping %s', response.status_code, url)
    except requests.RequestException:
        logging.error('error exceptions while scraping %s', url, exc_info=True)


def scrape_index(page):
    index_url = f'{BASE_URL}/page/{page}'
    return scrape_page(index_url)


def scrape_detail(detail_id):
    # detail_url = f'{BASE_URL}/detail/{detail_id}'
    return scrape_page(detail_id)


# 解析首页html
def parse_index(html):
    # pattern = re.compile('<a.*?href="(.*?)".*?class="name">', re.S)
    pattern = re.compile('<a.*?href="(.*?)".*?class="name">')
    items = re.findall(pattern, html)
    if not items:
        return []
    for item in items:
        # detail_url=urljoin(BASE_URL,item)
        detail_url = BASE_URL + '' + item
        logging.info('get detail url %s', detail_url)
        yield detail_url


def parse_detail(html):
    print('begin parse detail...')
    cover_pattern = re.compile('<img.*?src="(.*?)".*?class="cover">', re.S)
    title_pattern = re.compile('<h2.*?>(.*?)</h2>')
    categories_pattern = re.compile('<button.*?category.*?<span>(.*?)</span>.*?</button>', re.S)
    published_pattern = re.compile('(\d{4}-\d{2}-\d{2}).*?上映')
    # address_pattern=re.compile('')
    info_pattern = re.compile('<div.*?drama.*?>.*?<p.*?>(.*?)</p>', re.S)
    score_pattern = re.compile('<p.*?score.*?>(.*?)</p>', re.S)
    cover = re.search(cover_pattern, html).group(1).strip() if re.search(cover_pattern, html) else None
    name = re.search(title_pattern, html).group(1).strip() if re.search(title_pattern, html) else None
    categories = re.search(categories_pattern, html).group(1).strip() if re.search(categories_pattern, html) else None
    published_at = re.search(published_pattern, html).group(1).strip() if re.search(published_pattern, html) else None
    drama = re.search(info_pattern, html).group(1).strip() if re.search(info_pattern, html) else None
    score = float(re.search(score_pattern, html).group(1).strip()) if re.search(score_pattern, html) else None
    return {
        'cover': cover,
        'name': name,
        'categories': categories,
        'published_at': published_at,
        'drama': drama,
        'score': score
    }


def main():
    index_html=scrape_index(page)


if __name__ == '__main__':
    main()
